@dataclass
class ToolCallRecord:
    tool_name: str
    tool_input: dict
    tool_output: str
    duration_ms: float

@dataclass
class AgentStep:
    step_number: int
    reasoning: Optional[str]
    tool_calls: list[ToolCallRecord]
    input_tokens: int = 0
    output_tokens: int = 0
    cost_usd: float = 0.0

@dataclass
class Trace:
    trace_id: str
    agent_name: str
    steps: list[AgentStep]
    status: str  # "running", "completed", "failed", "loop_detected"
    total_cost_usd: float = 0.0

class AgentTracer:
    def start_trace(self, agent_name, query, model) -> str:
        """Start a new trace. Returns trace_id."""

    def log_step(self, trace_id, step: AgentStep):
        """Log a completed step — accumulates tokens and cost."""

    def end_trace(self, trace_id, output, status="completed"):
        """Mark trace as complete."""

    def get_trace_json(self, trace_id) -> str:
        """Export trace as JSON for debugging."""

    def print_summary(self, trace_id):
        """Human-readable trace summary."""

for step in range(max_steps):
    # ... get LLM response, extract tool calls ...

    for tool_call in tool_calls:
        # Check BEFORE executing
        loop_check = loop_detector.check_tool_call(
            tool_call.name, str(tool_call.arguments)
        )

        if loop_check.is_looping:
            # Inject warning into conversation instead of executing
            messages.append({
                "role": "tool",
                "content": f"LOOP DETECTED: {loop_check.message}"
            })
            break  # Skip remaining tool calls this step

from litellm import completion_cost

response = completion(model="gpt-4o", messages=messages)

# Get cost from the response
cost = completion_cost(completion_response=response)
print(f"This step cost: ${cost:.4f}")

class AgentEvaluator:
    def evaluate_one(self, question, agent_answer, reference_answer):
        response = completion(
            model=self.judge_model,
            messages=[
                {"role": "system", "content": EVALUATION_PROMPT},
                {"role": "user", "content": f"""
                    QUESTION: {question}
                    REFERENCE ANSWER: {reference_answer}
                    AGENT'S ANSWER: {agent_answer}
                """}
            ],
            response_format={"type": "json_object"},
            temperature=0,  # Deterministic evaluation
        )
        scores = json.loads(response.choices[0].message.content)
        return EvaluationResult(**scores)

[
  {
    "question": "What is the population of Tokyo?",
    "reference": "The population of Tokyo is approximately 14 million...",
    "category": "factual",
    "difficulty": "easy"
  },
  {
    "question": "Compare renewable energy policies of EU and US",
    "reference": "The EU has committed to... while the US...",
    "category": "comparison",
    "difficulty": "hard"
  }
]